# -*- coding: utf-8 -*-
"""
    @author:sirius
    @time:2017.10.14
"""
import pandas as pd
import numpy as np
import math
import time

def read_mdc():
    """
        提取application文件'db_key','name'字段并存储到目标文件
        'db_key':数据主键
        'name':应用名称
    """
    app_data = pd.read_csv('./data/application.csv', sep='\t')
    
    app_data = app_data[app_data['event'] == 'Application.Started']
    app_data = app_data.loc[:, ['db_key', 'name']]    
    app_data = app_data[app_data['name'] != ' ']

    app_data.to_csv('./md_cache/application.csv', index=False)

    print 'Application-Finished...'

    """
        根据'db_key'字段结合records文件'userid','time'字段并存储到目标文件
        'userid':用户ID
        'time':事件时间戳
    """
    app_data = pd.read_csv('./md_cache/application.csv')
    # reader = pd.read_csv('./data/records.csv', sep = '\t', iterator = True)
    rec_data = pd.read_csv('./data/records.csv')

    # 分块读取大文件
    # loop = True
    # chunkSize = 100000
    # chunks = []
    # num = 0
    # while loop:
    #     try:
    #         if num < 10:
    #             chunk = reader.get_chunk(chunkSize)
    #             chunks.append(chunk)
    #             num += 1
    #         else :
    #             loop = False
    #             print "Loop is stopped."    
    #     except StopIteration:
    #         loop = False
    #         print "Iteration is stopped."
    # rec_data = pd.concat(chunks, ignore_index=True)

    rec_data = rec_data.loc[:, ['db_key','userid', 'time']]  
    # 组合application文件数据和records文件数据
    data = pd.merge(app_data, rec_data, how='left', on='db_key')

    for i,j in data.iterrows():
        if math.isnan(j['time']):
            data.drop(i,axis=0,inplace=True)

    data.to_csv('./md_cache/app_rec_data.csv', index=False)

    print 'App_Rec-Finished...'

    """
        根据'time'字段结合gps文件'latitude','longitude','speed'并存储到目标文件
        'latitude':纬度
        'longitude':经度
        'speed':速度传感器
    """
    app_rec_data = pd.read_csv('./md_cache/app_rec_data.csv')

    gps_data = pd.read_csv('./data/gps.csv', sep='\t')
    gps_data = gps_data.loc[:, ['time', 'latitude', 'longitude', 'speed']]

    # 组合application文件数据、records文件数据和gps文件数据
    data = pd.merge(app_rec_data, gps_data, how='left', on='time')
    data.to_csv('./md_cache/app_rec_gps.csv', index=False)

    print 'App_Rec_Gps-Finished...'

    # 读取合并APP和GPS后的数据
    # data = pd.read_csv('./md_cache/app_gps_data.csv')

    # 根据是否含有GPS数据作为是否有传感器的值
    # sensor_list = []
    # for i in data['longitude']:
    #     if str(i) == str(np.nan):
    #         sensor_list.append(0)
    #     else:
    #         sensor_list.append(1)
    # data['sensor'] = sensor_list

    # 将Na值用0取代
    # data.loc[data['sensor'] == 0, ['latitude', 'longitude']] = 0
    # data = data[['time', 'sensor', 'latitude', 'longitude', 'name']]
    # data.to_csv('./md_cache/result.csv', index=False)

    # 读取带传感器值的数据
    # data = pd.read_csv('./md_cache/app_gps_sensor.csv')

    # TODO 取维度除以经度作为location字段，后期使用枚举方式
    # location_list = []
    # for i, j in data.iterrows():
        # if j['longitude'] == 0:
          # location_list.append(0)
        # else:
            # location_list.append(j['latitude'] / j['longitude'])

    # 将除后的值进行截取
    # for i, j in enumerate(location_list):
        # location_list[i] = float(str(j)[:5])

    # 添加字段
    # data['location'] = location_list
    # data = data['time', 'sensor', 'location', 'name']
    # data.to_csv('./md_cache/result.csv', index=False)

    # 最终清洗后的数据
    # data = pd.read_csv('./md_cache/result.csv')
    # print(data)

if __name__ == '__main__':
    read_mdc()
